#!/usr/bin/perl
#
# Countclean - count the number of occurences of a
# value in the Clean part of a database.
#
# Will give output in the format:
#
# //database:fieldname
# total:count
# empty:count
# "value":count
# "value":count
# Quotes in value are printed as \"
#
# Values containing commas are counted multiple times.
#
# Arguments: filename field...
#

use POSIX;
use FileTable;

$reportlimit = 1;

while ($ARGV[0] =~ /^-/) {
    $opt = shift;
    if ($opt eq "-raw") {
	$rawmode = 1;
    } elsif ($opt eq "-n") {
	$maxrec = shift(@ARGV) + 0;
	print STDERR "Counting max $maxrec records\n";
    } elsif ($opt eq "-limit") {
	$reportlimit = shift(@ARGV) + 0;
	print STDERR "Reporting counts of $reportlimit and above\n";
    }
}
$filename = shift;
@fields = @ARGV;
$file = FileTable::open("$filename");
die "Unable to open DB $filename\n" if !$file;

while ($key = $file->each) {
    if ($rawmode) {
	$rec = $file->get($key);
    } else {
	$rec = $file->getclean($key);
    }
    for $field (@fields) {
	if ($rec->{$field}) {
	    @values = split(/\s*,\s*/, $rec->{$field});
	    for $value (@values) {
		++ $counts{$field}{$value};
		++ $countsx{$field}{"values"};
	    }
	    ++ $countsx{$field}{"total"};
	} else {
	    ++ $countsx{$field}{"empty"};
	}
    }
    ++ $reccount;
    last if $maxrec && $reccount >= $maxrec;
    if ($reccount % 100 == 0 && -t STDERR) {
	print STDERR "$reccount\r";
    }
}
print STDERR "$reccount records counted\n";
for $field (keys(%counts)) {
    print "//$filename:$field\n";
    for $statistic (keys(%{$countsx{$field}})) {
	print "$statistic:",$countsx{$field}{$statistic}, "\n";
    }
    $others = 0;
    for $value (sort {$counts{$field}{$b} <=> $counts{$field}{$a}}
		keys(%{$counts{$field}})) {
	if ($counts{$field}{$value} >= $reportlimit) {
	    ($valq = $value) =~ s/"/\\"/g;
	    print "\"$valq\":$counts{$field}{$value}\n";
	} else {
	    $others += $counts{$field}{$value};
	}
    }
    print "\"others\":$others\n" if $others;
}
